﻿using AngleSharp.Parser.Html;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;

namespace StatisticsWords
{
    class Program
    {
        static void Main(string[] args)
        {

            //KnownDistinct();
            string url = @"C:\Users\cnryb\Desktop\docs-master";

            string body;
            if (url.StartsWith("http"))
            {
                WebClient webClient = new WebClient();
                string html = webClient.DownloadString(url);

                var parser = new HtmlParser();
                var document = parser.Parse(html);

                body = document.QuerySelector("body").TextContent.ToLower();
            }
            else if (File.Exists(url))
            {
                body = File.ReadAllText(url);
            }
            else
            {
                body = GetDirAllFileContent(url);
            }

            //\W
            body = Regex.Replace(body, @"\W", " ");
            body = Regex.Replace(body, @"\d", " ");
            //var words = body.Split(' ');
            var words = Regex.Split(body, @"\s+");
            Dictionary<string, int> dicWords = new Dictionary<string, int>();
            foreach (var kk in words)
            {
                string item = kk.ToLower();
                if (dicWords.ContainsKey(item))
                {
                    dicWords[item] += 1;
                }
                else
                {
                    dicWords.Add(item, 1);
                }
            }
            var knowns = File.ReadAllLines("Known.txt");
            foreach (var kk in knowns)
            {
                string k = kk.ToLower();
                if (dicWords.ContainsKey(k))
                    dicWords.Remove(k);
            }

            var sortDic = dicWords.OrderBy(d => d.Value);
            var resPath = "res.txt";
            if (File.Exists(resPath))
                File.Delete(resPath);
            List<string> res = new List<string>();
            foreach (var item in sortDic)
            {
                string line = $"{item.Value.ToString("000")}\t{item.Key}";
                Console.WriteLine(line);
                res.Add(line);
            }
            File.WriteAllLines(resPath, res);
            Console.WriteLine(sortDic.Count());
            Console.Read();
            
        }

        static string GetDirAllFileContent(string dirPath)
        {
            DirectoryInfo di = new DirectoryInfo(dirPath);

            StringBuilder sb = new StringBuilder();
            foreach (var item in di.GetDirectories())
            {
                sb.Append(GetDirAllFileContent(item.FullName));
            }

            foreach (var item in di.GetFiles("*.md"))
            {
                sb.Append(File.ReadAllText(item.FullName));
            }
            return sb.ToString();
        }


        static void KnownDistinct()
        {
            DirectoryInfo di = new DirectoryInfo(Environment.CurrentDirectory);
            string path = Path.Combine(di.Parent.Parent.FullName, "Known.txt");
            var lines = File.ReadAllLines(path);
            lines = lines.Distinct(new StringComparer()).ToArray();
            File.WriteAllLines(path, lines);
        }
    }

    class StringComparer : IEqualityComparer<string>
    {
        public bool Equals(string x, string y)
        {
            return x == y;
        }



        public int GetHashCode(string obj)
        {
            int hash = 0;
            if (string.IsNullOrEmpty(obj)) return hash;
            var cs = obj.ToCharArray();
            for (int i = 0; i < cs.Length; i++)
            {
                hash = hash + i + cs[i];
            }
            return hash;
        }
    }
}
